{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第3步：再次调整弱分类器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "import seaborn as sns\n",
    "\n",
    "from numpy import nan as NaN\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Filled_Form_N</th>\n",
       "      <th>Filled_Form_Y</th>\n",
       "      <th>Device_Type_Mobile</th>\n",
       "      <th>Device_Type_Web-browser</th>\n",
       "      <th>Mobile_Verified_N</th>\n",
       "      <th>Mobile_Verified_Y</th>\n",
       "      <th>Source_S122</th>\n",
       "      <th>Source_S123</th>\n",
       "      <th>Source_S124</th>\n",
       "      <th>Source_S127</th>\n",
       "      <th>...</th>\n",
       "      <th>Loan_Amount_Submitted</th>\n",
       "      <th>Loan_Tenure_Submitted</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>DOB_month</th>\n",
       "      <th>DOB_year</th>\n",
       "      <th>age</th>\n",
       "      <th>Lead_Creation_Date_month</th>\n",
       "      <th>Lead_Creation_Date_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>620000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.99</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8</td>\n",
       "      <td>1987</td>\n",
       "      <td>32</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>260000.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>33.00</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1993</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>6600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1978</td>\n",
       "      <td>41</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.50</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1985</td>\n",
       "      <td>34</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>16.25</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1975</td>\n",
       "      <td>44</td>\n",
       "      <td>7</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Filled_Form_N  Filled_Form_Y  Device_Type_Mobile  Device_Type_Web-browser  \\\n",
       "0              0              1                   1                        0   \n",
       "1              1              0                   1                        0   \n",
       "2              1              0                   0                        1   \n",
       "3              1              0                   0                        1   \n",
       "4              1              0                   0                        1   \n",
       "\n",
       "   Mobile_Verified_N  Mobile_Verified_Y  Source_S122  Source_S123  \\\n",
       "0                  0                  1            1            0   \n",
       "1                  0                  1            1            0   \n",
       "2                  0                  1            1            0   \n",
       "3                  1                  0            1            0   \n",
       "4                  0                  1            1            0   \n",
       "\n",
       "   Source_S124  Source_S127           ...             Loan_Amount_Submitted  \\\n",
       "0            0            0           ...                          620000.0   \n",
       "1            0            0           ...                          260000.0   \n",
       "2            0            0           ...                          100000.0   \n",
       "3            0            0           ...                          200000.0   \n",
       "4            0            0           ...                          300000.0   \n",
       "\n",
       "   Loan_Tenure_Submitted  Interest_Rate  Processing_Fee  Disbursed  DOB_month  \\\n",
       "0                    4.0          13.99          3100.0        0.0          8   \n",
       "1                    4.0          33.00          2000.0        0.0          2   \n",
       "2                    5.0          28.50          6600.0        0.0          2   \n",
       "3                    3.0          28.50          5000.0        0.0          6   \n",
       "4                    5.0          16.25          7000.0        0.0          4   \n",
       "\n",
       "   DOB_year  age  Lead_Creation_Date_month  Lead_Creation_Date_year  \n",
       "0      1987   32                         7                     2015  \n",
       "1      1993   26                         7                     2015  \n",
       "2      1978   41                         7                     2015  \n",
       "3      1985   34                         7                     2015  \n",
       "4      1975   44                         7                     2015  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('FE_X_train.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Disbursed属性的不同取值和出现的次数\n",
      "0.0    4694\n",
      "1.0     264\n",
      "Name: Disbursed, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#对类别型特征，观察其取值范围及直方图\n",
    "categorical_features = ['Disbursed']\n",
    "for col in categorical_features:\n",
    "    print('\\n%s属性的不同取值和出现的次数'%col)\n",
    "    print(train[col].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#再次按比例随机抽样，（机器配置太低）\n",
    "import random as rd\n",
    "import math as ma\n",
    "\n",
    "def typeicalSampling(group, typeicalFracDict):\n",
    "    name = group.name\n",
    "    frac = typeicalFracDict[name]\n",
    "    return group.sample(frac=frac)\n",
    "def group_sample(data_set,lable,typeicalFracDict):\n",
    "    #分层抽样\n",
    "    #data_set数据集\n",
    "    #lable分层变量名\n",
    "    #typeicalFracDict：分类抽样比例\n",
    "    gbr=data_set.groupby(by=[lable])\n",
    "    result=data_set.groupby(lable,group_keys=False).apply(typeicalSampling,typeicalFracDict)\n",
    "    return result\n",
    "\n",
    "data = train\n",
    "\n",
    "data_set=data\n",
    "label='Disbursed'\n",
    "\n",
    "typicalFracDict = {\n",
    "    0: 0.54747,\n",
    "    1: 0.2073\n",
    "}\n",
    "train=group_sample(data_set,label,typicalFracDict)\n",
    "# print(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Disbursed属性的不同取值和出现的次数\n",
      "0.0    2570\n",
      "1.0      55\n",
      "Name: Disbursed, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#对类别型特征，观察其取值范围及直方图\n",
    "categorical_features = ['Disbursed']\n",
    "for col in categorical_features:\n",
    "    print('\\n%s属性的不同取值和出现的次数'%col)\n",
    "    print(train[col].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sns.countplot(train.Disbursed);\n",
    "# pyplot.xlabel('Disbursed');\n",
    "# pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['Disbursed']\n",
    "X_train = train.drop([\"Disbursed\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, cv_folds, early_stopping_rounds=10,useTrainCV=True):\n",
    "    \n",
    "    if useTrainCV:   \n",
    "        xgb_param = alg.get_xgb_params()\n",
    "#         xgb_param['num_class'] = 9\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        boost_round = alg.get_params()['n_estimators']\n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=boost_round, nfold =cv_folds,\n",
    "                         metrics='error', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print (cvresult)\n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('my_preds0.1_2_0.001_14.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-error-mean']\n",
    "        test_stds = cvresult['test-error-std'] \n",
    "        \n",
    "        train_means = cvresult['train-error-mean']\n",
    "        train_stds = cvresult['train-error-std'] \n",
    "\n",
    "#         x_axis = range(0, n_estimators)\n",
    "#         pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "#         pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "#         pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "#         pyplot.xlabel( 'n_estimators' )\n",
    "#         pyplot.ylabel( 'Log Loss' )\n",
    "#         pyplot.savefig( 'n_estimators4_2_3_699.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='logloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    print(\"n_estimators=\", n_estimators)\n",
    "    \n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print (logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   train-error-mean  train-error-std  test-error-mean  test-error-std\n",
      "0          0.022286         0.001103         0.023238        0.006962\n",
      "1          0.021524         0.002029         0.021714        0.004908\n",
      "2          0.020952         0.001313         0.020952        0.005251\n",
      "n_estimators= 3\n",
      "logloss of train :\n",
      "0.4779289752301716\n",
      "[0.97904762 0.97904762 0.97904762 0.97904762 0.97904762]\n",
      "CV logloss: 97.90% (0.00%)\n"
     ]
    }
   ],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,4)\n",
    "xgb2_3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=14,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=2,\n",
    "        min_child_weight=0.001,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'binary:logistic',\n",
    "#         num_class = 9,\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb2_3, X_train, y_train, cv_folds = 5)\n",
    "from sklearn.model_selection import cross_val_score\n",
    "results = cross_val_score(xgb2_3, X_train, y_train, cv=kfold)\n",
    "print (results)\n",
    "print(\"CV logloss: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cvresult = pd.DataFrame.from_csv('my_preds4_2_3_699.csv')\n",
    "\n",
    "# cvresult = cvresult.iloc[100:]\n",
    "# # plot\n",
    "# test_means = cvresult['test-mlogloss-mean']\n",
    "# test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "# train_means = cvresult['train-mlogloss-mean']\n",
    "# train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "# x_axis = range(100,cvresult.shape[0]+100)\n",
    "        \n",
    "# fig = pyplot.figure(figsize=(10, 10), dpi=100)\n",
    "# pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "# pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "# pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "# pyplot.xlabel( 'n_estimators' )\n",
    "# pyplot.ylabel( 'Log Loss' )\n",
    "# pyplot.savefig( 'n_estimators_detail4_2_3_699.png' )\n",
    "\n",
    "# pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV logloss: 97.90% (0.00%)\n"
     ]
    }
   ],
   "source": [
    "print(\"CV logloss: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train-error-mean</th>\n",
       "      <th>train-error-std</th>\n",
       "      <th>test-error-mean</th>\n",
       "      <th>test-error-std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_estimators</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.022286</td>\n",
       "      <td>0.001103</td>\n",
       "      <td>0.023238</td>\n",
       "      <td>0.006962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.021524</td>\n",
       "      <td>0.002029</td>\n",
       "      <td>0.021714</td>\n",
       "      <td>0.004908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.020952</td>\n",
       "      <td>0.001313</td>\n",
       "      <td>0.020952</td>\n",
       "      <td>0.005251</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              train-error-mean  train-error-std  test-error-mean  \\\n",
       "n_estimators                                                       \n",
       "0                     0.022286         0.001103         0.023238   \n",
       "1                     0.021524         0.002029         0.021714   \n",
       "2                     0.020952         0.001313         0.020952   \n",
       "\n",
       "              test-error-std  \n",
       "n_estimators                  \n",
       "0                   0.006962  \n",
       "1                   0.004908  \n",
       "2                   0.005251  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('my_preds0.1_2_0.001_14.csv')\n",
    "cvresult"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
